package jarvis.fs.analyzer;

import java.io.Reader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
/**
 * 
 * Function: The analyzer to tokenize the word
 * which can remove stop words and bad words
 * also do stemming use snowball(see http://snowball.tartarus.org/)
 * @author Jarvis.Guo
 *
 */
public class WordAnalyzer extends Analyzer{

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub

	}

	@Override
	public TokenStream tokenStream(String fieldName, Reader reader) {
		//general tokenize
		TokenStream result = new StandardTokenizer(reader);
	    result = new StandardFilter(result);
	    //to lower case
	    result = new LowerCaseFilter(result);
	    //stop words remove
	    result = new StopFilter(result, StandardAnalyzer.STOP_WORDS);
	    //do stemming
	    result = new SnowballFilter(result, "English");
	    //remove bad words whose length is 1 or be a float
	    result = new BadWordsFilter(result);
	    return result;
	}

}
